package com.g2room.driving.grap.life;

import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.alibaba.fastjson.JSON;
import com.g2room.driving.grap.GenericGrap;
import com.g2room.myuall.grap.ky.model.po.GrapErr;
import com.g2room.myuall.grap.univ.po.Article;
import com.g2room.myuall.util.DbUtil;
import com.g2room.myuall.util.HtmlUtil;
import com.g2room.myuall.util.StringUtil;

public class GrapWaduanziData implements Runnable {
	public static String db = "life_grap";
	private static  String yddBaseUrl = "http://www.yikexun.cn";
	private static  String source = "挖段子";
	private static  Map<String, Integer> errUrls = new HashMap<String, Integer>();
	private static  List<String> detailUrls = new ArrayList<String>();
	private  List<String> listUrls = new ArrayList<String>();
	private static Map<String, String> listUrlMap = new HashMap<String, String>();
	static {
//		listUrlMap.put("http://www.yikexun.cn/mingrenjingdianyulu/","名人经典语录");
		listUrlMap.put("http://www.yikexun.cn/niandujingdianyulu/","年度经典语录");
		listUrlMap.put("http://www.yikexun.cn/feizhuliuyulu/","非主流经典语录");
		listUrlMap.put("http://www.yikexun.cn/aiqingyulu/","爱情语录");
		listUrlMap.put("http://www.yikexun.cn/jingdiangaoxiaoyulu/","经典搞笑语录");
		listUrlMap.put("http://www.yikexun.cn/2009jingdianyulu/","2009经典语录大全");
		listUrlMap.put("http://www.yikexun.cn/2010jingdianyulu/","2010经典语录");
		listUrlMap.put("http://www.yikexun.cn/2011jingdianyulu/","2011经典语录");
		listUrlMap.put("http://www.yikexun.cn/2012jingdianyulu/","2012经典语录");
		listUrlMap.put("http://www.yikexun.cn/niandujingdianyulu/2013yulu/","2017经典语录");
		listUrlMap.put("http://www.yikexun.cn/niandujingdianyulu/2014yulu/","2014经典语录");
		listUrlMap.put("http://www.yikexun.cn/niandujingdianyulu/","年度经典语录");
		listUrlMap.put("http://www.yikexun.cn/feizhuliuyulu/","非主流经典语录");
		listUrlMap.put("http://www.yikexun.cn/mingrenjingdianyulu/","名人经典语录");
		listUrlMap.put("http://www.yikexun.cn/aiqingyulu/","爱情语录");
//		listUrlMap.put("http://www.yikexun.cn/jingdiangaoxiaoyulu/","经典搞笑语录");
		listUrlMap.put("http://www.yikexun.cn/maozedong/","毛泽东语录");
		listUrlMap.put("http://www.yikexun.cn/guojingming/","郭敬明经典语录");
		listUrlMap.put("http://www.yikexun.cn/zhangailing/","张爱玲经典语录");
		listUrlMap.put("http://www.yikexun.cn/zhangxiaoxian/","张小娴经典语录");
		listUrlMap.put("http://www.yikexun.cn/hanhan/","韩寒经典语录");
		listUrlMap.put("http://www.yikexun.cn/xiaoshenyang/","小沈阳经典语录");
		listUrlMap.put("http://www.yikexun.cn/wdqcszz/","我的青春谁做主");
		listUrlMap.put("http://www.yikexun.cn/niandujingdianyulu/","年度经典语录");
		listUrlMap.put("http://www.yikexun.cn/feizhuliuyulu/","非主流经典语录");
		listUrlMap.put("http://www.yikexun.cn/mingrenjingdianyulu/","名人经典语录");
		listUrlMap.put("http://www.yikexun.cn/aiqingyulu/","爱情语录");
		listUrlMap.put("http://www.yikexun.cn/jingdiangaoxiaoyulu/","经典搞笑语录");
		listUrlMap.put("http://www.yikexun.cn/niandujingdianyulu/","年度经典语录");
		listUrlMap.put("http://www.yikexun.cn/feizhuliuyulu/","非主流经典语录");
		listUrlMap.put("http://www.yikexun.cn/mingrenjingdianyulu/","名人经典语录");
		listUrlMap.put("http://www.yikexun.cn/aiqingyulu/","爱情语录");
		listUrlMap.put("http://www.yikexun.cn/jingdiangaoxiaoyulu/","经典搞笑语录");
		
		
		
		listUrlMap.put("http://www.yikexun.cn/mrmy/lizhi/","励志名言");
		listUrlMap.put("http://www.yikexun.cn/mrmy/rensheng/","人生格言");
		listUrlMap.put("http://www.yikexun.cn/mrmy/dushu/","读书名言");
		listUrlMap.put("http://www.yikexun.cn/mrmy/aiqing/","爱情名言");
		listUrlMap.put("http://www.yikexun.cn/mrmy/aiguo/","爱国名言");
		listUrlMap.put("http://www.yikexun.cn/mrmy/","名人名言");
		
		listUrlMap.put("http://www.yikexun.cn/jingdiantaici/","经典台词");
		listUrlMap.put("http://www.yikexun.cn/lizhi/qianming/","励志签名");
		listUrlMap.put("http://www.yikexun.cn/rensheng/ganwu/","人生感悟");
		listUrlMap.put("http://www.yikexun.cn/jingdianduanxin/","经典短信");
//		listUrlMap.put("http://www.yikexun.cn/duanzi/","段子");
		listUrlMap.put("http://www.yikexun.cn/QQ/","说说");
		
		
//		listUrlMap.put("http://www.yikexun.cn/shi/","诗句");
//		listUrlMap.put("http://www.yikexun.cn/suibi/","情感日志");
	}
	
	public static void main(String[] args) throws Exception{
//		GrapWaduanziData gl = new GrapWaduanziData();
//		for(String url : listUrlMap.keySet()) {
//			System.out.println("开始抓="+url);
//			gl.getYddPage(url);
//		}
		
		
		// 347
		
		getSourceUrl();
		GrapWaduanziData gl = new GrapWaduanziData();
//	 		for(int i=347; i< 1771; ) {
//	 			gl.getWdzList("http://www.waduanzi.com/joke/page/" + i);
//	 			i++;
////	 			for(int j = 0; j < 36 && i < 1771; j ++) {
////	 				gl.listUrls.add("http://www.waduanzi.com/joke/page/" + i);
////	 				i++;
////	 			}
////	        	Thread t = new Thread(gl);
////	        	t.start();
//	        }
	 		getErrUrl();
	 		for(String url : errUrls.keySet()) {
	 			System.out.println(url);
	 			switch(errUrls.get(url)) {
	 			case 23:
	 				gl.getWdzList(url);
	 				break;
	 			case 24:
	 				if(!detailUrls.contains(url)) {
						detailUrls.add(url);
						gl.getWaduanziDetail(url);
					}
	 				break;
	 			}
	 		}
	}
	
	public Document getDoc(String url, String errCode) {
		// 获取分页信息
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (IOException e) {
			e.printStackTrace();
			System.out.println(url + "======" + e.getMessage());
			GrapErr ge = new  GrapErr();
			ge.setUrl(url);
			ge.setType(errCode);
			ge.setDescription("错误："+e.getMessage());
			insert2ErrDb(ge);
		}
		return doc;
	}

	/**
	 * 获取一点点分页
	 */
	public void getYddPage(String url) {
		// 获取分页信息
		Document doc = getDoc(url, "1");
		if(doc == null) return;
		Elements els = doc.select(".page .pagelist li a");
		List<String> purls = new ArrayList<String>();
		if(null != els && !els.isEmpty()) {
			// list_11_100.html
			String href = els.get(els.size() -1).attr("href");
			if(null == href) return;
			String lastPage = href.substring(href.lastIndexOf("_") + 1);
			lastPage = lastPage.substring(0, lastPage.indexOf("."));
			href = href.replace(lastPage + ".html", "");
			for(int i = 1; i<= Integer.valueOf(lastPage) ; i++) {
				purls.add(url + href + i + ".html");
			}
		}
		for(String purl : purls) {
			sleep();
			getYddList(purl);
			System.out.println("抓完="+purl);
		}
	}
	

	/**
	 * 获取一点点列表
	 */
	public void getYddList(String url) {
		// 获取分页信息
		Document doc = getDoc(url, "2");
		if(doc == null) return;
		Elements els = doc.select(".art-ul li div.art-t h3 a");
		List<String> durls = new ArrayList<String>();
		if(null != els && !els.isEmpty()) {
			for(Element el : els) {
				String href = yddBaseUrl + el.attr("href");
				if(!detailUrls.contains(href)) {
					durls.add(href);
					detailUrls.add(href);
				}
			}
		}
		List<Article> artis = new ArrayList<Article>();
        for(String durl : durls) {
        	sleep();
        	Article art = getYddDetail(durl);
        	if(null != art) artis.add(art);
        }
        if(artis.size() > 0) insertArtiDb(artis, "life_jok_arti");
	}
	

	public Article getYddDetail(String url) {
		// 获取分页信息
		Document doc = getDoc(url, "3");
		if(doc == null) return null;
		Article arti = new Article();
		Elements els = doc.select(".nowsite .place a");
		// 获取类
		if(null != els && !els.isEmpty()) {
			switch (els.size()) {
			case 4:
				arti.setClassName(els.get(3).text());
			case 3:
				arti.setTypeName(els.get(2).text());
			case 2:
				arti.setCategoryName(els.get(1).text());
			}
		}
		
		els = doc.select(".arc-title h1");
		if(null != els && !els.isEmpty()) {
			arti.setTitle(els.first().text());
		}
		els= doc.select(".neirong");
		if(null == els || els.isEmpty()) return null;
		
		els = els.first().children();
		StringBuffer content = new StringBuffer();
		for(Element e : els) {
			if(e.tagName().equalsIgnoreCase("div") && e.attr("class").startsWith("ad")) {
				e.remove();
			} else {
				content.append(e.outerHtml());
			}
		}
		els = doc.select(".arc-page .pagelist li a");
		if(null != els && !els.isEmpty()) {
			// 24363_5.html
			String href = els.get(els.size() - 2).attr("href");
			String lastPage = href.substring(href.lastIndexOf("_") + 1);
			lastPage = lastPage.substring(0, lastPage.indexOf("."));
			for(int i=2; i< Integer.parseInt(lastPage)+1; i++) {
				String c = getPageDetail(url.replace(".html", "_"+i + ".html"));
				if(null == c) return null;
				content.append(c);
			}
		}
		arti.setContent(HtmlUtil.delATag(content.toString().replaceAll("/uploads/", "http://www.yikexun.cn/uploads/")));
		arti.setSourceUrl(url);
		arti.setSource("一点点");
		return arti;
	}
	
	public String getPageDetail(String url) {
		Document doc = getDoc(url, "4");
		if(doc == null) return null;
		Elements els= doc.select(".neirong");
		if(null == els || els.isEmpty()) return null;
		
		els = els.first().children(); 
		StringBuffer content = new StringBuffer();
		for(Element e : els) {
			if(e.tagName().equalsIgnoreCase("div") && e.attr("class").startsWith("ad")) {
			} else {
				content.append(e.outerHtml());
			}
		}
		// 处理另外一种情况
		if(content.length() < 5) {
			els= doc.select(".arc-content table tbody tr td");
			els.select("div.ad-box650").remove();
			els.select("div.neirong").remove();
			return els.first().html();
		}
		return content.toString();
	}
	
	
	/**
	 * 获取列表列表
	 */
	public void getWdzList(String url) {
		// 获取分页信息
		Document doc = getDoc(url, "23");
		if(doc == null) return;
		Elements els = doc.select(".post-line-list div.post-item div.item-detail .item-title a");
		List<String> durls = new ArrayList<String>();
		if(null != els && !els.isEmpty()) {
			for(Element el : els) {
				if(!detailUrls.contains(el.attr("href"))) {
					durls.add(el.attr("href"));
					detailUrls.add(el.attr("href"));
				}
			}
		}
		List<Article> artis = new ArrayList<Article>();
        for(String durl : durls) {
        	Article art = getWaduanziDetail(durl);
        	if(null != art) artis.add(art);
        }
        if(artis.size() > 0) insertArtiDb(artis, "life_jok");
	}
	
	public Article getWaduanziDetail(String url) {
		// 获取分页信息
		Document doc = getDoc(url, "24");
		if(doc == null) return null;
		Article arti = new Article();
		Elements els = doc.select(".post-content .post-tags");
		if(null != els && !els.isEmpty()) {
			StringBuffer tags = new StringBuffer();
			for(Element del : els) {
				if(tags.length() > 1) tags.append(",");
				tags.append(del.text());
			}
			arti.setCover(tags.toString());
		}
		els = doc.select("title");
		// 开车等红灯中，突觉某地方奇痒无比（你懂的, 你懂的,妹纸 - 挖段子网
		if(null != els && !els.isEmpty()) {
			arti.setTitle(els.first().text().replace(" - 挖段子网", "").replace("挖段子网", ""));
		}
		Element el = doc.getElementById("item-content");
		if(null == el) return null;
		
		els = el.children(); 
		for(Element e : els) {
			if(e.tagName().equalsIgnoreCase("br") || e.tagName().equalsIgnoreCase("div")) {
				e.remove();
			}
		}
		arti.setContent(el.html());
		arti.setSourceUrl(url);
		arti.setSource(source);
		return arti;
	}
	
	public void insertArtiDb(List<Article> artis, String tb) {
		String sql = "insert into "+tb+"(category_name,type_name,content,source_url,class_name,source,title,publish_time,cover,cover_height,cover_width, state,summary) values(?,?,?,?,?,?,?,?,?,?,?,?,?)";
		PreparedStatement ps = null;
		try {
			ps = DbUtil.getConn(db).prepareStatement(sql);
			for(Article cc : artis) {
				ps.setString(1, cc.getCategoryName());
				ps.setString(2, cc.getTypeName());
				ps.setString(3, cc.getContent());
				ps.setString(4, cc.getSourceUrl());
				ps.setString(5, cc.getClassName());
				ps.setString(6, cc.getSource());
				ps.setString(7, cc.getTitle());
				ps.setString(8, cc.getPublishTime());
				ps.setInt(10, cc.getCoverHeight());
				ps.setInt(11, cc.getCoverWidth());
				ps.setString(9, cc.getCover());
				ps.setInt(12, cc.getState());
				ps.setString(13, cc.getSummary());
				ps.addBatch();
			}
			ps.executeBatch();	
		} catch (Exception e) {
			e.printStackTrace();
			GrapErr ge = new  GrapErr();
			ge.setType("5");
			ge.setContent(JSON.toJSONString(artis));
			ge.setDescription("获取DB错误");
			insert2ErrDb(ge);
		
		} finally {
			if(null != ps) {
				try {
					ps.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

	public void insert2ErrDb(GrapErr ge) {
		PreparedStatement grapPs = null;
		try {
			grapPs = DbUtil.getConn(db).prepareStatement("insert into grap_err(description,url,content,type) values(?,?,?,?)");
			grapPs.setString(1, ge.getDescription());
			grapPs.setString(2, ge.getUrl());
			grapPs.setString(3, ge.getContent());
			grapPs.setString(4, ge.getType());
			grapPs.execute();
			grapPs.close();
		} catch(Exception e) {
			e.printStackTrace();
		} finally {
			if(null != grapPs) {
				try {
					grapPs.close();
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

	@Override
	public void run() {
	        for(String lurl : listUrls) {
	        	getWdzList(lurl);
	        }
	}
	

	public static void getSourceUrl() {
		try {
			Statement statement = DbUtil.getConn(db).createStatement();
			String sql = "select source_url from life_jok";
			ResultSet rs = statement.executeQuery(sql);
			while(rs.next()) {
				detailUrls.add(rs.getString("source_url"));
			}
			rs.close();
			statement.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	


	public static void getErrUrl() {
		try {
			Statement statement = DbUtil.getConn(db).createStatement();
			String sql = "select url, type from grap_err where type>20";
			ResultSet rs = statement.executeQuery(sql);
			while(rs.next()) {
				errUrls.put(rs.getString("url"), rs.getInt("type"));
			}
			rs.close();
			statement.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}

	public void sleep() {
    	try {
    		Thread.sleep(Integer.parseInt(StringUtil.genRandomNum(1)) * 100);
    	} catch (Exception e) {
    	}
	}
}
